package raphaelMatcher;

import static raphaelMatcher.CleanAttributeValuesSchema.cav_articleId;
import static raphaelMatcher.CleanAttributeValuesSchema.cav_attributeValueText;
import static raphaelMatcher.CleanAttributeValuesSchema.cav_attributesNamesId;
import static raphaelMatcher.CleanAttributeValuesSchema.cav_segments;
import static raphaelMatcher.CreateSentencesSchema.cs_articleId;
import static raphaelMatcher.CreateSentencesSchema.cs_sentence;
import static raphaelMatcher.CreateSentencesSchema.cs_sentenceId;
import static raphaelMatcher.CreateSentencesSchema.cs_tokenPos;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;

//import wikiextract.data.util.StringUtil;
//import wikiextract.util.io.BinaryWriter;
//import wikiextract.util.io.DelimitedReader;

/*
 * Match attribute values with sentences.
 */
public class CreateMatchesStanford {

	static String dataDir = "O:\\unix\\projects\\pardosa\\s2\\raphaelh\\data\\all";
	static String tmpDir = "O:\\unix\\projects\\pardosa\\s2\\raphaelh\\data\\tmp";
	static String wrtDir = "O:\\unix\\projects\\pardosa\\s2\\clzhang\\raphaelh";
	// static String dataDir;
	// static String tmpDir;

	static String input1Sorted = "attributes.cleaned.sortedByArticleId";
	static String input2Sorted = "sentences.stanford.tokenized.sortedByArticleId";
	static String output1 = "congle.trainingset.stanford.bin.sortedByArticleId"; // output
																					// sorted
																					// by
																					// articleId

	public static void main(String[] args) throws Exception {
		readArguments(args);
		input1Sorted = dataDir + "/" + input1Sorted;
		input2Sorted = dataDir + "/" + input2Sorted;
		output1 = wrtDir + "/" + output1;

		int page = 0;
		// BinaryWriter writer2 = new BinaryWriter(output1);

		DelimitedReader reader1 = new DelimitedReader(input1Sorted);
		DelimitedReader reader2 = new DelimitedReader(input2Sorted);
		DelimitedWriter writer2 = new DelimitedWriter(output1);
		String[] t1 = reader1.read();
		String[] t2 = reader2.read();

		List<String[]> block1 = new ArrayList<String[]>();
		List<String[]> block2 = new ArrayList<String[]>();

		// attributes for certain page
		block1.add(t1);
		while ((t1 = reader1.read()) != null
				&& t1[cav_articleId].equals(block1.get(0)[cav_articleId]))
			block1.add(t1);

		// sentences for certain page
		block2.add(t2);
		while ((t2 = reader2.read()) != null
				&& t2[cs_articleId].equals(block2.get(0)[cs_articleId]))
			block2.add(t2);

		while (!block1.isEmpty() && !block2.isEmpty()) {
			int c = Integer.parseInt(block1.get(0)[cav_articleId])
					- Integer.parseInt(block2.get(0)[cs_articleId]);

			if (c < 0) {
				// move to attributes for next page
				block1.clear();
				if (t1 != null) {
					block1.add(t1);
					while ((t1 = reader1.read()) != null
							&& t1[cav_articleId]
									.equals(block1.get(0)[cav_articleId]))
						block1.add(t1);
				}
			} else if (c > 0) {
				// move to sentences for next page
				block2.clear();
				if (t2 != null) {
					block2.add(t2);
					while ((t2 = reader2.read()) != null
							&& t2[cs_articleId]
									.equals(block2.get(0)[cs_articleId]))
						block2.add(t2);
				}
			} else {
				System.out.println(page++);
				// we have attributes and sentences for same page, output merged
				// tuples
				// (we now have a textId, and a block of all sentences, and a
				// block of all attributes, for that textId)

				// sort attributes by attributeNamesId
				Collections.sort(block1, new Comparator<String[]>() {
					public int compare(String[] o1, String[] o2) {
						return Integer.parseInt(o1[cav_attributesNamesId])
								- Integer.parseInt(o2[cav_attributesNamesId]);
					}
				});
				// sort sentences by sentenceId
				Collections.sort(block2, new Comparator<String[]>() {
					public int compare(String[] o1, String[] o2) {
						return Integer.parseInt(o1[cs_sentenceId])
								- Integer.parseInt(o2[cs_sentenceId]);
					}
				});

				// explore all combinations of attributes and sentences
				for (int i = 0; i < block1.size(); i++) {
					for (int j = 0; j < block2.size(); j++) {
						String[] s1 = block1.get(i);
						String[] s2 = block2.get(j);

						// fields for infobox attributes
						String attributeValueText = s1[cav_attributeValueText];
						String segments = s1[cav_segments];

						// fields for sentences
						String sentence = s2[cs_sentence];
						String tokenPos = s2[cs_tokenPos];

						String[] sgmts = StringUtil.split(segments, '|', 24);

						List<int[]> matchPos = null;
						{
							for (int sgmt = 0; sgmt < sgmts.length; sgmt++) {
								if (sgmts[sgmt].trim().length() == 0)
									continue;

								// only consider left segments for now
								if (sgmt > 0)
									break;

								matchPos = indexesOfTokenSequence(sentence,
										tokenPos, attributeValueText,
										sgmts[sgmt]);

								// ensure that we can use bytes for storage
								for (Iterator<int[]> it = matchPos.iterator(); it
										.hasNext();) {

									int[] mp = it.next();
									if (mp[0] >= 0) {
										System.out.print("");
									}
									if (mp[1] > 127)
										it.remove();
								}
							}
						}

						// if we have less than 2 matchPos we fill up with -1
						if (matchPos == null)
							matchPos = new ArrayList<int[]>(2);
						while (matchPos.size() < 2)
							matchPos.add(new int[] { -1, -1 });

						System.out.print(Integer.parseInt(s2[cs_sentenceId])
								+ "\t" + s1[cav_articleId] + "\t"
								+ s1[cav_attributesNamesId] + "\t");
						for (byte b = 0; b < 2; b++) {
							System.out.print((byte) matchPos.get(b)[0] + "\t");
							System.out.print((byte) matchPos.get(b)[1] + "\t");
						}
						System.out.println();
						writer2.write(Integer.parseInt(s2[cs_sentenceId]) + "");
						writer2.write(Integer.parseInt(s1[cav_articleId]) + "");
						writer2.write(Integer
								.parseInt(s1[cav_attributesNamesId])
								+ "");
						for (byte b = 0; b < 2; b++) {
							writer2.write((byte) matchPos.get(b)[0] + "");
							writer2.write((byte) matchPos.get(b)[1] + "");
						}
					}
				}

				// pick next blocks
				block1.clear();
				if (t1 != null) {
					block1.add(t1);
					while ((t1 = reader1.read()) != null
							&& t1[1].equals(block1.get(0)[1]))
						block1.add(t1);
				}
				block2.clear();
				if (t2 != null) {
					block2.add(t2);
					while ((t2 = reader2.read()) != null
							&& t2[2].equals(block2.get(0)[2]))
						block2.add(t2);
				}
			}
		}

		reader2.close();
		reader1.close();
		writer2.close();
	}

	public static List<int[]> indexesOfTokenSequence(String text1,
			String tokens1, String text2, String tokens2) {
		String[] toks1 = StringUtil.split(tokens1, ' ', 256);
		String[] toks2 = StringUtil.split(tokens2, ' ', 256);
		List<int[]> result = new ArrayList<int[]>();

		for (int i = 0; i < toks1.length - toks2.length; i++) {
			boolean matches = true;
			for (int j = 0; j < toks2.length; j++) {
				String[] tok1 = StringUtil.split(toks1[i + j], ':', 2);
				String[] tok2 = StringUtil.split(toks2[j], ':', 2);

				String t1 = text1.substring(Integer.parseInt(tok1[0]), Integer
						.parseInt(tok1[1]));
				String t2 = text2.substring(Integer.parseInt(tok2[0]), Integer
						.parseInt(tok2[1]));

				if (!t1.equals(t2)) {
					matches = false;
					break;
				}
			}
			if (matches)
				result.add(new int[] { i, i + toks2.length });
		}
		return result;
	}

	private static void readArguments(String[] args) {
		for (int i = 0; i < args.length; i++) {
			if ("-dataDir".equals(args[i]))
				dataDir = args[++i];
			else if ("-tmpDir".equals(args[i]))
				tmpDir = args[++i];
		}

		if (dataDir == null || tmpDir == null) {
			System.out.println("usage: java "
					+ CreateMatchesStanford.class.getName() + "\n"
					+ "             -dataDir DATA_DIR\n"
					+ "             -tmpDir TMP_DIR");
			System.exit(1);
		}
	}
}
